package usi.ch.onto.lucene.scoring;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;


public class KeyTermExtractor {

	private boolean buildCompleteList;
	private List<IndexTermItem> completeList;
	private ArrayList<String> termList;
	private ArrayList<String> goodIdfList;

	/**
	 * Class that reads the Lucene index
	 * 
	 * @return ArayList of terms, after applying some thresholds on term
	 *         frequencies
	 * 
	 **/
	public KeyTermExtractor() {
		buildCompleteList = false;
		completeList = null;
	}

	public KeyTermExtractor(boolean buildList) {
		buildCompleteList = true;
		completeList = new ArrayList<IndexTermItem>();
	}

	public void retrieveTerms(File indexdir) throws IOException {
		this.termList = new ArrayList<String>();
		this.goodIdfList = new ArrayList<String>();

		Directory fSDirectory = FSDirectory.open(indexdir);
		IndexReader iReader = IndexReader.open(fSDirectory);
		FilterIndexReader fReader = new FilterIndexReader(iReader);
		Similarity sim = new DefaultSimilarity();

		int numDocs = fReader.numDocs();

		TermEnum terms = fReader.terms();
		while (terms.next()) {
			Term term = terms.term();
			TermDocs termdocs = fReader.termDocs(term);
			int df = fReader.docFreq(term);

			while (termdocs.next()) {
				int freq = termdocs.freq(); // term frequency with this document
				float idf = sim.idf(df, numDocs);// idf of term for this
													// document
				float tf = sim.tf(freq);
				float tfidf = (tf * idf);

				if (buildCompleteList) {
					IndexTermItem iti = new IndexTermItem();
					iti.doc = termdocs.doc();
					iti.freq = freq;
					iti.idf = idf;
					iti.tf = tf;
					iti.tfidf = tfidf;
					iti.term = term.text();
					iti.df = df;
					iti.field = term.field();

					if (freq > 4 & term.text().length() >= 4) {
						completeList.add(iti);
					}

				}

				if (freq > 3 & term.text().length() > 3) {
					// System.out.println(term.text() + " idf:  " + idf +
					// " tf: " + tf + "  tfidf  " + tfidf);
					goodIdfList.add(term.text());
				}
			}
			termdocs.close();
			String termText = term.text();
			if (df > 2 && termText.length() > 3) {
				termList.add(termText);
			}
		}
		iReader.close();

	}

	public ArrayList<String> getTermList() {
		return termList;
	}

	public ArrayList<String> getGoodIdfList() {
		return goodIdfList;

	}

	public List<IndexTermItem> getCompleteList() {
		return completeList;
	}

	public Collection<IndexTermItem> getTopTermList(float topPercentage){
		List<IndexTermItem> topList = new ArrayList<IndexTermItem>();
		if (buildCompleteList && completeList != null && completeList.size() > 0) {
			Collections.sort(completeList, TFIDF_ORDER);

			int currentDoc = -1;
			float maxTfidf = -1; 
			
			//fills the listOfMaxTfidfProDoc. each item has a unique docid, and the maxtfidf for that doc
			for(IndexTermItem item : completeList){
				if (item.doc != currentDoc) {
					currentDoc = item.doc; 
					maxTfidf = item.tfidf;
				}
				//all items that have tfidf >= maxtfidf * (topPercentage / 100)	
				if (item.tfidf >= maxTfidf * (topPercentage / 100))
						topList.add(item);
			}
		
		} else {
			System.out
					.println("you need to call retrieveTerms and use the constructor IndexTermReaderBig(boolean buildList) or list is empty! ");
		}

		return topList;
	}
	
	//gets the maxtfidf for current doc
	//considers that the list is ordered by doc and tfidf
	//, so the first item that matches the doc has the max tfidf 
	private float getMaxTfIdf(List<IndexTermItem> list, int doc){
		float maxTfIdf = -1;

		for(int i = 0; i < completeList.size(); i++){
			if (doc == completeList.get(i).doc){
				maxTfIdf = completeList.get(i).tfidf;
				break;
			}
		}
		
		return maxTfIdf;
	}

	//if you change this method and the order is affected the getTopTermList might return wrong values
	//because it relies that the max tfidf is at the top of each different doc
	static final Comparator<IndexTermItem> TFIDF_ORDER = new Comparator<IndexTermItem>() {
		//orders by doc ASC and tfidf DESC 
		public int compare(IndexTermItem e1, IndexTermItem e2) {
			int cmp = (e1.doc < e2.doc ? - 1 : (e1.doc == e2.doc ? 0 : 1));
			if (cmp == 0)
				cmp = (e1.tfidf > e2.tfidf ? - 1 : (e1.tfidf == e2.tfidf ? 0 : 1)); 
			return cmp;
		}
	};
}

